package com.allen.flink.dbus.increments;

import com.alibaba.otter.canal.protocol.FlatMessage;
import com.allen.flink.dbus.function.DbusProcessFunction;
import com.allen.flink.dbus.model.Flow;
import com.allen.flink.dbus.schema.FlatMessageSchema;
import com.allen.flink.dbus.sink.HBaseSyncSink;
import com.allen.flink.dbus.source.FlowSource;
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.BroadcastStream;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;

import java.util.Properties;

/**
 * 功能:实施增量同步模块
 *
 * @date: 2020-03-26 16:10
 * @author: Allen
 * @version: 0.0.4-snapshot
 * @Email: allenZyhang@163.com
 * @since: JDK 1.8
 **/
public class IncrementSyncApp {
    public static final MapStateDescriptor<String, Flow> flowStateDescriptor = new MapStateDescriptor<String, Flow>("flowBroadCastState", BasicTypeInfo.STRING_TYPE_INFO,
        TypeInformation.of(new TypeHint<Flow>() {
        }));
    
    public static void main(String[] args) throws Exception {
        //获取执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        
        //配置
        Properties props = new Properties();
        props.put("bootstrap.servers", "master:9092");//kafka地址
        props.put("zookeeper.connect", "master:2181");//zookeeper地址
        props.put("group.id", "group1");//组
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("auto.offset.reset", "latest");
        props.put("flink.partition-discovery.interval-millis", "30000");
        
        //消费kafka数据
        FlinkKafkaConsumer010<FlatMessage> myConsumer = new FlinkKafkaConsumer010<>("test", new FlatMessageSchema(), props);
        DataStream<FlatMessage> message = env.addSource(myConsumer);
        
        //同库,同表数据进入同一个分组，一个分区  解决MQ时序性问题
        KeyedStream<FlatMessage, String> keyedMessage = message.keyBy(new KeySelector<FlatMessage, String>() {
            @Override
            public String getKey(FlatMessage value) throws Exception {
                return value.getDatabase() + value.getTable();
            }
        });
        
        //读取配置流
        BroadcastStream<Flow> broadcast = env.addSource(new FlowSource()).broadcast(flowStateDescriptor);
        
        //连接数据流和配置流
        DataStream<Tuple2<FlatMessage, Flow>> connectedStream = keyedMessage.connect(broadcast).process(
            new DbusProcessFunction()).setParallelism(1);
    
        //输出到HBase
        connectedStream.addSink(new HBaseSyncSink());
    
        env.execute("IncrementSyncApp");
    }
    
}
